package com.djhu.newscollector.crawel.imp.wangyi;

import com.djhu.newscollector.core.Holder.JobHolder;
import com.djhu.newscollector.core.dispatch.JobDispatcher;
import com.djhu.newscollector.core.entity.Response;
import com.djhu.newscollector.crawel.core.PageProcess;
import com.djhu.newscollector.crawel.core.Url;
import com.djhu.newscollector.crawel.imp.worker.NewsResponse;
import com.djhu.newscollector.crawel.imp.worker.WangyiNewsWorker;
import com.djhu.newscollector.crawel.net.DefaultHttpVisitor;
import com.djhu.newscollector.crawel.net.HttpRequest;
import com.djhu.newscollector.crawel.net.HttpVisitor;
import com.djhu.newscollector.crawel.net.Page;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * Created by zw on 2018/2/12.
 */
public class LinkPageProcess implements PageProcess {
    private JobHolder jobHolder;

    private JobDispatcher jobDispatcher;
    private static Logger logger = LogManager.getLogger(WangyiNewsWorker.class.getName());


    @Override
    public Response process(Url url) {
        NewsResponse newsResponse = new NewsResponse();
        if (url instanceof Url) {
            HttpRequest request = new HttpRequest();
            request.setUrl(url);
            HttpVisitor httpVisitor = new DefaultHttpVisitor();
            try {
                Page page = httpVisitor.visit(request);
                Document doc = page.doc();
                Elements links = doc.select("a[href]");
                for (Element element : links) {
                    //System.out.println(element.attr("abs:href"));
                    String linkUrl = element.attr("abs:href");
                    if (linkUrl.contains("163")) {
                        Url crawlerUrl = new Url();
                        crawlerUrl.setUrl(linkUrl);
                        if (jobHolder.contains(crawlerUrl)) {
                            logger.warn("linkUrl has existed !!" + linkUrl);
                            continue;
                        }
                        jobDispatcher.retJob(crawlerUrl);
                        logger.info("add a new link ,url is " + linkUrl);
                    }
                }
                newsResponse.setPage(page);
            } catch (Exception e) {
                newsResponse.setException(e);
            }
        }else {
            newsResponse.setException(new RuntimeException("type error"));
        }
        return newsResponse;
    }

    public JobHolder getJobHolder() {
        return jobHolder;
    }

    public void setJobHolder(JobHolder jobHolder) {
        this.jobHolder = jobHolder;
    }

    public JobDispatcher getJobDispatcher() {
        return jobDispatcher;
    }

    public void setJobDispatcher(JobDispatcher jobDispatcher) {
        this.jobDispatcher = jobDispatcher;
    }
}